#!/usr/bin/python
# -*- coding:utf-8 -*-
# Orignal Author: FrankHacker
# Modified by Linson @691000737
# Require requests module, python3.x+
# 处理常见违禁词，全站扫描，违禁词可以从自己的网站导出成json格式，替换13行内容
# 程序会在当前目录生成badword.txt文件
import requests
import re,json

#要扫描的初始网址
url = "http://www.baidu.com"

#-----------违禁词处理 START-----------
d=json.loads('[{"bid":"1","replacefrom":"\u7eaf\u5929\u7136","replaceto":"","deny":"0"}, {"bid":"2","replacefrom":"\u6539\u5584\u7761\u7720","replaceto":"","deny":"0"}, {"bid":"3","replacefrom":"\u56fd\u5bb6\u7ea7","replaceto":"","deny":"0"}, {"bid":"4","replacefrom":"\u4e16\u754c\u7ea7","replaceto":"","deny":"0"}, {"bid":"5","replacefrom":"\u6700\u9ad8\u7ea7","replaceto":"","deny":"0"}, {"bid":"6","replacefrom":"\u6700\u4f73","replaceto":"","deny":"0"}, {"bid":"7","replacefrom":"\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"9","replacefrom":"\u9996\u4e2a","replaceto":"","deny":"0"}, {"bid":"10","replacefrom":"\u6700\u597d","replaceto":"","deny":"0"}, {"bid":"11","replacefrom":"\u7cbe\u786e","replaceto":"","deny":"0"}, {"bid":"12","replacefrom":"\u9876\u7ea7","replaceto":"","deny":"0"}, {"bid":"13","replacefrom":"\u6700\u4f4e","replaceto":"","deny":"0"}, {"bid":"14","replacefrom":"\u6700\u5e95","replaceto":"","deny":"0"}, {"bid":"15","replacefrom":"\u6700","replaceto":"","deny":"0"}, {"bid":"16","replacefrom":"\u6700\u4fbf\u5b9c","replaceto":"","deny":"0"}, {"bid":"17","replacefrom":"\u6700\u5927\u7a0b\u5ea6","replaceto":"","deny":"0"}, {"bid":"18","replacefrom":"\u6700\u65b0\u6280\u672f","replaceto":"","deny":"0"}, {"bid":"19","replacefrom":"\u6700\u5148\u8fdb\u79d1\u5b66","replaceto":"","deny":"0"}, {"bid":"20","replacefrom":"\u56fd\u5bb6\u7ea7\u4ea7\u54c1","replaceto":"","deny":"0"}, {"bid":"21","replacefrom":"\u586b\u8865\u56fd\u5185\u7a7a\u767d","replaceto":"","deny":"0"}, {"bid":"22","replacefrom":"\u7edd\u5bf9","replaceto":"","deny":"0"}, {"bid":"23","replacefrom":"\u72ec\u5bb6","replaceto":"","deny":"0"}, {"bid":"24","replacefrom":"\u9996\u5bb6","replaceto":"","deny":"0"}, {"bid":"25","replacefrom":"\u6700\u65b0","replaceto":"","deny":"0"}, {"bid":"26","replacefrom":"\u6700\u5148\u8fdb","replaceto":"","deny":"0"}, {"bid":"27","replacefrom":"\u7b2c\u4e00\u54c1\u724c","replaceto":"","deny":"0"}, {"bid":"28","replacefrom":"\u91d1\u724c","replaceto":"","deny":"0"}, {"bid":"29","replacefrom":"\u540d\u724c","replaceto":"","deny":"0"}, {"bid":"30","replacefrom":"\u6700\u8d5a","replaceto":"","deny":"0"}, {"bid":"31","replacefrom":"\u8d85\u8d5a","replaceto":"","deny":"0"}, {"bid":"32","replacefrom":"\u6700\u5148","replaceto":"","deny":"0"}, {"bid":"33","replacefrom":"\u5de8\u661f","replaceto":"","deny":"0"}, {"bid":"34","replacefrom":"\u5962\u4f88","replaceto":"","deny":"0"}, {"bid":"35","replacefrom":"\u81f3\u5c0a","replaceto":"","deny":"0"}, {"bid":"36","replacefrom":"\u9876\u7ea7\u4eab\u53d7","replaceto":"","deny":"0"}, {"bid":"37","replacefrom":"\u56fd\u5bb6","replaceto":"","deny":"0"}, {"bid":"38","replacefrom":"\u8d28\u91cf\u514d\u68c0","replaceto":"","deny":"0"}, {"bid":"39","replacefrom":"\u65e0\u9700\u56fd\u5bb6\u8d28\u91cf\u68c0\u6d4b","replaceto":"","deny":"0"}, {"bid":"40","replacefrom":"\u514d\u62bd\u68c0","replaceto":"","deny":"0"}, {"bid":"41","replacefrom":"\u6700\u9ad8","replaceto":"","deny":"0"}, {"bid":"42","replacefrom":"\u5bb6\u5ead\u5fc5\u5907","replaceto":"","deny":"0"}, {"bid":"43","replacefrom":"\u795e\u4e39","replaceto":"","deny":"0"}, {"bid":"44","replacefrom":"\u6291\u5236","replaceto":"","deny":"0"}, {"bid":"45","replacefrom":"\u5947\u6548","replaceto":"","deny":"0"}, {"bid":"46","replacefrom":"\u795e\u6548","replaceto":"","deny":"0"}, {"bid":"47","replacefrom":"\u6700\u5148","replaceto":"","deny":"0"}, {"bid":"48","replacefrom":"\u56fd\u5bb6\u673a\u5173","replaceto":"","deny":"0"}, {"bid":"49","replacefrom":"\u9ad8\u6548","replaceto":"","deny":"0"}, {"bid":"50","replacefrom":"\u8d85\u8d5a","replaceto":"","deny":"0"}, {"bid":"51","replacefrom":"\u7956\u4f20","replaceto":"","deny":"0"}, {"bid":"52","replacefrom":"\u6700\u4f73","replaceto":"","deny":"0"}, {"bid":"53","replacefrom":"\u6700\u8584","replaceto":"","deny":"0"}, {"bid":"54","replacefrom":"\u5408\u9a70\u540d\u5546\u6807","replaceto":"","deny":"0"}, {"bid":"55","replacefrom":"\u795b\u7600","replaceto":"","deny":"0"}, {"bid":"56","replacefrom":"\u901f\u6548","replaceto":"","deny":"0"}, {"bid":"57","replacefrom":"\u56fd\u5bb6\u514d\u68c0","replaceto":"","deny":"0"}, {"bid":"58","replacefrom":"\u795e\u4ed9","replaceto":"","deny":"0"}, {"bid":"59","replacefrom":"\u4e2d\u56fd\u9a70\u540d","replaceto":"","deny":"0"}, {"bid":"60","replacefrom":"\u6700\u9002","replaceto":"","deny":"0"}, {"bid":"61","replacefrom":"\u8d85\u5f3a","replaceto":"","deny":"0"}, {"bid":"62","replacefrom":"\u6700\u7f8e","replaceto":"","deny":"0"}, {"bid":"63","replacefrom":"\u7279\u6548","replaceto":"","deny":"0"}, {"bid":"64","replacefrom":"\u5962\u4f88","replaceto":"","deny":"0"}, {"bid":"65","replacefrom":"\u6700\u5c0f","replaceto":"","deny":"0"}, {"bid":"66","replacefrom":"\u79d8\u5236","replaceto":"","deny":"0"}, {"bid":"67","replacefrom":"\u6700\u65b0\u6280\u672f","replaceto":"","deny":"0"}, {"bid":"68","replacefrom":"\u6700\u4fbf\u5b9c","replaceto":"","deny":"0"}, {"bid":"69","replacefrom":"100%","replaceto":"","deny":"0"}, {"bid":"70","replacefrom":"\u6700\u6d41\u884c","replaceto":"","deny":"0"}, {"bid":"71","replacefrom":"\u7ec8\u6781","replaceto":"","deny":"0"}, {"bid":"72","replacefrom":"\u51a0\u519b","replaceto":"","deny":"0"}, {"bid":"73","replacefrom":"\u524d\u65e0\u53e4\u4eba","replaceto":"","deny":"0"}, {"bid":"74","replacefrom":"\u6781\u81f4","replaceto":"","deny":"0"}, {"bid":"75","replacefrom":"\u65f6\u5c1a\u6700\u4f4e\u4ef7","replaceto":"","deny":"0"}, {"bid":"76","replacefrom":"\u6700\u5927","replaceto":"","deny":"0"}, {"bid":"77","replacefrom":"\u5168\u56fd\u9500\u91cf\u51a0\u519b","replaceto":"","deny":"0"}, {"bid":"78","replacefrom":"\uff08\u9065\u9065\uff09\u9886\u5148","replaceto":"","deny":"0"}, {"bid":"79","replacefrom":"\u8d28\u91cf\u514d\u68c0","replaceto":"","deny":"0"}, {"bid":"80","replacefrom":"\u4e2d\u56fd\u9a70\u540d\u5546\u6807","replaceto":"","deny":"0"}, {"bid":"81","replacefrom":"\u9996\u4e2a","replaceto":"","deny":"0"}, {"bid":"82","replacefrom":"\u8457\u540d","replaceto":"","deny":"0"}, {"bid":"83","replacefrom":"\u9886\u5bfc\u8005","replaceto":"","deny":"0"}, {"bid":"84","replacefrom":"\u9886\u8896\u54c1\u724c","replaceto":"","deny":"0"}, {"bid":"85","replacefrom":"\u4e13\u4f9b","replaceto":"","deny":"0"}, {"bid":"86","replacefrom":"\u4ec5\u6b64\u4e00\u6b21","replaceto":"","deny":"0"}, {"bid":"87","replacefrom":"\u6700\u65f6\u5c1a","replaceto":"","deny":"0"}, {"bid":"88","replacefrom":"\u6781\u54c1","replaceto":"","deny":"0"}, {"bid":"89","replacefrom":"\u6700\u65b0\u79d1\u6280","replaceto":"","deny":"0"}, {"bid":"90","replacefrom":"\u56fd\u9645\u54c1\u8d28","replaceto":"","deny":"0"}, {"bid":"91","replacefrom":"\u8d44\u6df1","replaceto":"","deny":"0"}, {"bid":"92","replacefrom":"\u7cbe\u51c6","replaceto":"","deny":"0"}, {"bid":"93","replacefrom":"\u8d2d\u7269\u5927\u8db4","replaceto":"","deny":"0"}, {"bid":"94","replacefrom":"\u6700\u65b0\u79d1\u5b66","replaceto":"","deny":"0"}, {"bid":"95","replacefrom":"\u6700\u597d","replaceto":"","deny":"0"}, {"bid":"96","replacefrom":"\u5168\u7f51\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"97","replacefrom":"\u6700\u4f4e\u4ef7","replaceto":"","deny":"0"}, {"bid":"98","replacefrom":"\u4e2d\u56fd\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"99","replacefrom":"\u5168\u7403\u7ea7","replaceto":"","deny":"0"}, {"bid":"100","replacefrom":"\u6781\u4f73\uff08\u7edd\u4f73\/\u7edd\u5bf9\uff09","replaceto":"","deny":"0"}, {"bid":"101","replacefrom":"\u6ca1\u6709\u4ed6\u5c31","replaceto":"","deny":"0"}, {"bid":"102","replacefrom":"\u4e07\u4eba\u75af\u62a2","replaceto":"","deny":"0"}, {"bid":"103","replacefrom":"\u6700\u65b0","replaceto":"","deny":"0"}, {"bid":"104","replacefrom":"\u586b\u8865\u56fd\u5185\u7a7a\u767d","replaceto":"","deny":"0"}, {"bid":"105","replacefrom":"\u8d81\u73b0\u5728","replaceto":"","deny":"0"}, {"bid":"106","replacefrom":"\u6700\u540e\u4e00\u6ce2","replaceto":"","deny":"0"}, {"bid":"107","replacefrom":"\u6700\u5148\u8fdb\u52a0\u5de5\u5de5\u827a","replaceto":"","deny":"0"}, {"bid":"108","replacefrom":"NO.1","replaceto":"","deny":"0"}, {"bid":"109","replacefrom":"\u5de8\u661f","replaceto":"","deny":"0"}, {"bid":"110","replacefrom":"\u6700\u7b26\u5408","replaceto":"","deny":"0"}, {"bid":"111","replacefrom":"\u7279\u6548","replaceto":"","deny":"0"}, {"bid":"112","replacefrom":"\u6700\u9ad8","replaceto":"","deny":"0"}, {"bid":"113","replacefrom":"\u53f2\u65e0\u524d\u4f8b","replaceto":"","deny":"0"}, {"bid":"114","replacefrom":"\u7eaf\u5929\u7136","replaceto":"","deny":"0"}, {"bid":"115","replacefrom":"\u9876\u7ea7\u4eab\u53d7","replaceto":"","deny":"0"}, {"bid":"116","replacefrom":"\u7279\u4f9b","replaceto":"","deny":"0"}, {"bid":"117","replacefrom":"\u91d1\u724c","replaceto":"","deny":"0"}, {"bid":"118","replacefrom":"\u5168\u56fd","replaceto":"","deny":"0"}, {"bid":"119","replacefrom":"\u70b9\u51fb\u9886\u5956","replaceto":"","deny":"0"}, {"bid":"120","replacefrom":"\u6700\u4f4e\u7ea7","replaceto":"","deny":"0"}, {"bid":"121","replacefrom":"\u6700\u5962\u4f88","replaceto":"","deny":"0"}, {"bid":"122","replacefrom":"\u62a2\u75af\u4e86","replaceto":"","deny":"0"}, {"bid":"123","replacefrom":"\u70b9\u51fb\u6709\u60ca\u559c","replaceto":"","deny":"0"}, {"bid":"124","replacefrom":"\u72ec\u5bb6","replaceto":"","deny":"0"}, {"bid":"125","replacefrom":"\u6700\u4f18\u79c0","replaceto":"","deny":"0"}, {"bid":"126","replacefrom":"\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"127","replacefrom":"\u4e16\u754c\u7ea7","replaceto":"","deny":"0"}, {"bid":"128","replacefrom":"\u5dc5\u5cf0","replaceto":"","deny":"0"}, {"bid":"129","replacefrom":"\u4e07\u80fd","replaceto":"","deny":"0"}, {"bid":"130","replacefrom":"\u6b63\u54c1","replaceto":"","deny":"0"}, {"bid":"131","replacefrom":"\u70b9\u51fb\u8bd5\u7a7f","replaceto":"","deny":"0"}, {"bid":"132","replacefrom":"\u8d85\u8d5a","replaceto":"","deny":"0"}, {"bid":"133","replacefrom":"\u65e0\u9700\u56fd\u5bb6\u8d28\u91cf\u68c0\u6d4b","replaceto":"","deny":"0"}, {"bid":"134","replacefrom":"\u9996\u9009","replaceto":"","deny":"0"}, {"bid":"135","replacefrom":"\u62a2\u7206","replaceto":"","deny":"0"}, {"bid":"136","replacefrom":"\u5927\u724c","replaceto":"","deny":"0"}, {"bid":"137","replacefrom":"\u606d\u559c\u83b7\u5956","replaceto":"","deny":"0"}, {"bid":"138","replacefrom":"\u9519\u8fc7\u5c31\u6ca1\u673a\u4f1a\u4e86","replaceto":"","deny":"0"}, {"bid":"139","replacefrom":"\u5168\u6c11\u75af\u62a2","replaceto":"","deny":"0"}, {"bid":"140","replacefrom":"\u6700\u8d5a","replaceto":"","deny":"0"}, {"bid":"141","replacefrom":"\u771f\u76ae","replaceto":"","deny":"0"}, {"bid":"142","replacefrom":"\u6700\u9ad8\u6863","replaceto":"","deny":"0"}, {"bid":"143","replacefrom":"\u9886\u53d6\u5956\u54c1","replaceto":"","deny":"0"}, {"bid":"144","replacefrom":"\u54c1\u724c\u56e2","replaceto":"","deny":"0"}, {"bid":"145","replacefrom":"\u81f3\u5c0a","replaceto":"","deny":"0"}, {"bid":"146","replacefrom":"\u5962\u4f88","replaceto":"","deny":"0"}, {"bid":"147","replacefrom":"\u6700","replaceto":"","deny":"0"}, {"bid":"148","replacefrom":"\u5168\u6c11\u514d\u5355","replaceto":"","deny":"0"}, {"bid":"149","replacefrom":"\u5168\u7f51\u9996\u53d1","replaceto":"","deny":"0"}, {"bid":"150","replacefrom":"\u5168\u6c11\u62a2\u8d2d","replaceto":"","deny":"0"}, {"bid":"151","replacefrom":"\u4e16\u754c\u9886\u5148","replaceto":"","deny":"0"}, {"bid":"152","replacefrom":"\u518d\u4e0d\u62a2\u5c31\u6ca1\u4e86","replaceto":"","deny":"0"}, {"bid":"153","replacefrom":"\u56fd\u5bb6\u7ea7\u4ea7\u54c1","replaceto":"","deny":"0"}, {"bid":"154","replacefrom":"\u9876\u7ea7\uff08\u9876\u5c16\/\u5c16\u7aef\uff09","replaceto":"","deny":"0"}, {"bid":"155","replacefrom":"\u9886\u8896","replaceto":"","deny":"0"}, {"bid":"156","replacefrom":"\u9876\u7ea7\u5de5\u827a","replaceto":"","deny":"0"}, {"bid":"157","replacefrom":"\u6700\u7231","replaceto":"","deny":"0"}, {"bid":"158","replacefrom":"\u4e13\u5bb6\u63a8\u8350","replaceto":"","deny":"0"}, {"bid":"159","replacefrom":"\u56fd\u5bb6\u9886\u5bfc\u4eba","replaceto":"","deny":"0"}, {"bid":"160","replacefrom":"\u6700\u53d7\u6b22\u8fce","replaceto":"","deny":"0"}, {"bid":"161","replacefrom":"\u5468\u5e74\u5e86","replaceto":"","deny":"0"}, {"bid":"162","replacefrom":"\u6700\u5927\u7a0b\u5ea6","replaceto":"","deny":"0"}, {"bid":"163","replacefrom":"\u4e2d\u56fd\u9a70\u540d\uff08\u9a70\u540d\u5546\u6807\uff09","replaceto":"","deny":"0"}, {"bid":"164","replacefrom":"\u9996\u6b21","replaceto":"","deny":"0"}, {"bid":"165","replacefrom":"\u968f\u65f6\u6da8\u4ef7","replaceto":"","deny":"0"}, {"bid":"166","replacefrom":"\u6700\u5148","replaceto":"","deny":"0"}, {"bid":"167","replacefrom":"\u4e00\u6d41","replaceto":"","deny":"0"}, {"bid":"168","replacefrom":"\u5168\u56fd\u9996\u53d1","replaceto":"","deny":"0"}, {"bid":"169","replacefrom":"\u9500\u91cf\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"170","replacefrom":"\u56fd\u5bb6\u7ea7","replaceto":"","deny":"0"}, {"bid":"171","replacefrom":"\u65e0\u654c","replaceto":"","deny":"0"}, {"bid":"172","replacefrom":"\u4e00\u5929","replaceto":"","deny":"0"}, {"bid":"173","replacefrom":"\u4eca\u5929","replaceto":"","deny":"0"}, {"bid":"174","replacefrom":"\u4f18\u79c0","replaceto":"","deny":"0"}, {"bid":"175","replacefrom":"\u6700\u8212\u9002","replaceto":"","deny":"0"}, {"bid":"176","replacefrom":"\u7b2c\u4e00\u54c1\u724c","replaceto":"","deny":"0"}, {"bid":"177","replacefrom":"\u7cbe\u54c1\u56e2","replaceto":"","deny":"0"}, {"bid":"178","replacefrom":"\u5c31","replaceto":"","deny":"0"}, {"bid":"179","replacefrom":"\u6700\u4fbf\u5b9c","replaceto":"","deny":"0"}, {"bid":"180","replacefrom":"\u9996\u6b3e","replaceto":"","deny":"0"}, {"bid":"181","replacefrom":"\u9996\u53d1","replaceto":"","deny":"0"}, {"bid":"182","replacefrom":"\u4ec5\u9650","replaceto":"","deny":"0"}, {"bid":"183","replacefrom":"\u5012\u8ba1\u65f6","replaceto":"","deny":"0"}, {"bid":"184","replacefrom":"\u6700\u5e95","replaceto":"","deny":"0"}, {"bid":"185","replacefrom":"\u7956\u4f20","replaceto":"","deny":"0"}, {"bid":"186","replacefrom":"\u514d\u62bd\u68c0","replaceto":"","deny":"0"}, {"bid":"187","replacefrom":"\u6700\u4f4e","replaceto":"","deny":"0"}, {"bid":"188","replacefrom":"\u738b\u724c","replaceto":"","deny":"0"}, {"bid":"189","replacefrom":"\u6700\u5148\u8fdb\u79d1\u5b66","replaceto":"","deny":"0"}, {"bid":"190","replacefrom":"\u5468\u672b","replaceto":"","deny":"0"}, {"bid":"191","replacefrom":"\u6700\u5148\u8fdb","replaceto":"","deny":"0"}, {"bid":"192","replacefrom":"\u72ec\u4e00\u65e0\u4e8c","replaceto":"","deny":"0"}, {"bid":"193","replacefrom":"\u6700\u5177","replaceto":"","deny":"0"}, {"bid":"194","replacefrom":"\u6700\u5148\u4eab\u53d7","replaceto":"","deny":"0"}, {"bid":"195","replacefrom":"\u6700\u4f73","replaceto":"","deny":"0"}, {"bid":"196","replacefrom":"\u8001\u5b57\u53f7","replaceto":"","deny":"0"}, {"bid":"197","replacefrom":"\u4e4b\u738b","replaceto":"","deny":"0"}, {"bid":"198","replacefrom":"\u5b87\u5b99\u7ea7","replaceto":"","deny":"0"}, {"bid":"199","replacefrom":"\u95ea\u8d2d","replaceto":"","deny":"0"}, {"bid":"200","replacefrom":"\u5355\u54c1\u56e2","replaceto":"","deny":"0"}, {"bid":"201","replacefrom":"\u70b9\u51fb\u83b7\u53d6","replaceto":"","deny":"0"}, {"bid":"202","replacefrom":"\u738b\u8005","replaceto":"","deny":"0"}, {"bid":"203","replacefrom":"\u70b9\u51fb\u8f6c\u8eab","replaceto":"","deny":"0"}, {"bid":"204","replacefrom":"\u79d2\u6740","replaceto":"","deny":"0"}, {"bid":"205","replacefrom":"\u638c\u95e8\u4eba","replaceto":"","deny":"0"}, {"bid":"206","replacefrom":"\u7f14\u9020\u8005","replaceto":"","deny":"0"}, {"bid":"207","replacefrom":"\u9886\u5148\u4e0a\u5e02","replaceto":"","deny":"0"}, {"bid":"208","replacefrom":"\u9ad8\u7ea7","replaceto":"","deny":"0"}, {"bid":"209","replacefrom":"TOP.1","replaceto":"","deny":"0"}, {"bid":"210","replacefrom":"\u9a6c\u4e0a\u964d\u4ef7\u4eca\u65e5","replaceto":"","deny":"0"}, {"bid":"211","replacefrom":"\u72ec\u5bb6\u914d\u65b9","replaceto":"","deny":"0"}, {"bid":"212","replacefrom":"\u6700\u540e","replaceto":"","deny":"0"}, {"bid":"213","replacefrom":"\u6700\u4f18","replaceto":"","deny":"0"}, {"bid":"214","replacefrom":"\u7279\u60e0\u8db4","replaceto":"","deny":"0"}, {"bid":"215","replacefrom":"\u9ad8\u6863","replaceto":"","deny":"0"}, {"bid":"216","replacefrom":"\u540d\u724c","replaceto":"","deny":"0"}, {"bid":"217","replacefrom":"\u6c38\u4e45","replaceto":"","deny":"0"}, {"bid":"218","replacefrom":"\u968f\u65f6\u7ed3\u675f","replaceto":"","deny":"0"}, {"bid":"219","replacefrom":"\u56fd\u5bb6(\u56fd\u5bb6\u514d\u68c0\uff09","replaceto":"","deny":"0"}, {"bid":"220","replacefrom":"\u70b9\u51fb\u7ffb\u8f6c","replaceto":"","deny":"0"}, {"bid":"221","replacefrom":"\u521b\u9886\u54c1\u724c","replaceto":"","deny":"0"}, {"bid":"222","replacefrom":"\u6700\u9ad8\u7ea7","replaceto":"","deny":"0"}, {"bid":"223","replacefrom":"\u51e0\u5929\u51e0\u591c","replaceto":"","deny":"0"}, {"bid":"224","replacefrom":"\u5356\u75af\u4e86","replaceto":"","deny":"0"}, {"bid":"225","replacefrom":"\u6392\u540d\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"226","replacefrom":"\u5168\u56fd\u7b2c\u4e00","replaceto":"","deny":"0"}, {"bid":"227","replacefrom":"\u72ec\u5bb6","replaceto":"","deny":"0"}, {"bid":"228","replacefrom":"\u6700\u805a\u62e2","replaceto":"","deny":"0"}]')
data=[]
for i in d:
    data.append(i['replacefrom'])

def handler(url,text):
    '''
    针对每个页面url和text(抓取的文本内容) 进行处理
    '''
    global data
    bad_f=open('./badword.txt','a+')
    for i in data:
        if i in text:
            bad_f.write(url+":"+i+"\n")
    bad_f.close()
#----------违禁词处理 END------------

def url_protocol(url):
    '''
    获取输入的url地址的协议，是http、https等
    '''
    return re.findall(r'.*(?=://)',url)[0]

urlprotocol = url_protocol(url)

def same_url(url):
    '''
    处理用户输入的url，并为后续判断是否为一个站点的url做准备，爬取的时候不能爬到其它站，那么爬取将无止境
    :return: sameurl
    '''
    #将完整的url中的http://删除
    url = url.replace(urlprotocol + '://','')
    #判断删除http://之后的url有没有www，如果没有就加上‘www.’，但不存储，
    #只是为了同化所有将要处理的url，都有了‘www.’之后，
    #就可以找以‘www.’开始的到第一个‘/’结束中的所有字符串作为该站的主域名
    if re.findall(r'^www',url) == []:
        sameurl = 'www.' + url
        if sameurl.find('/') != -1:
            sameurl = re.findall(r'(?<=www.).*?(?=/)', sameurl)[0]
        else:
            sameurl = sameurl + '/'
            sameurl = re.findall(r'(?<=www.).*?(?=/)', sameurl)[0]
    else:
        if url.find('/') != -1:
            sameurl = re.findall(r'(?<=www.).*?(?=/)', url)[0]
        else:
            sameurl = url + '/'
            sameurl = re.findall(r'(?<=www.).*?(?=/)', sameurl)[0]
    return sameurl

domain_url = same_url(url)

'''
处理url的类，对已访问过的和未访问过的进行记录，待后续使用
'''
class linkQuence:
    def __init__(self):
        self.visited = []    #已访问过的url初始化列表
        self.unvisited = []  #未访问过的url初始化列表

    def getVisitedUrl(self):  #获取已访问过的url
        return self.visited
    def getUnvisitedUrl(self):  #获取未访问过的url
        return self.unvisited
    def addVisitedUrl(self,url):  #添加已访问过的url
        return self.visited.append(url)
    def addUnvisitedUrl(self,url):   #添加未访问过的url
        if url != '' and url not in self.visited and url not in self.unvisited:
            return self.unvisited.insert(0,url)

    def removeVisited(self,url):
        return self.visited.remove(url)
    def popUnvisitedUrl(self):    #从未访问过的url中取出一个url
        try:                      #pop动作会报错终止操作，所以需要使用try进行异常处理
            return self.unvisited.pop()
        except:
            return None
    def unvisitedUrlEmpty(self):   #判断未访问过列表是不是为空
        return len(self.unvisited) == 0

class Spider():
    '''
    真正的爬取程序
    '''
    def __init__(self,url):
        self.linkQuence = linkQuence()   #引入linkQuence类
        self.linkQuence.addUnvisitedUrl(url)   #并将需要爬取的url添加进linkQuence对列中
        self.current_deepth = 1    #设置爬取的深度
        '''
        这里需要注意:
        爬取分为：***先深度后广度***和***先广度和后深度***
        1、如果是先深度后广度，那么给定一个url，然后从其页面中的任意一个可用链接进行深度爬取，很可能无限至循环下去
        （在处理不当的时候，但一般情况下大家都会处理的很好，无非是判断哪些url是已经爬取过的，哪些是新爬取到的url）
        2、如果是先广度后深度，就是将一个url页面中的所有链接进行爬取，然后分类处理过滤
        （在这种处理不当的时候也会出现无限循环的可能，但一般情况下大家都会处理的很好，无非是判断哪些url是已经爬取过的，哪些是新爬取到的url）
        '''
    def getPageLinks(self,url):
        '''
        获取页面中的所有链接
        '''
        try:
            pageSource = requests.get(url).text
            pageLinks = re.findall(r'(?<=href=\").*?(?=\")|(?<=href=\').*?(?=\')',pageSource)
            #处理违禁词
            handler(url,pageSource)
            return pageLinks
        except:
            return []

    def processUrl(self,url):
        '''
        判断正确的链接及处理相对路径为正确的完整url
        :return:
        '''
        true_url = []
        for l in self.getPageLinks(url):
            if re.findall(r'/',l):
                if re.findall(r':',l):
                    true_url.append(l)
                else:
                    true_url.append(urlprotocol + '://' + domain_url + l)
        return true_url

    def sameTargetUrl(self,url):
        '''
        判断是否为同一站点链接，防止爬出站外，然后导致无限尝试爬取
        '''
        same_target_url = []
        for l in self.processUrl(url):
            if re.findall(domain_url,l):
                same_target_url.append(l)
        return same_target_url

    def unrepectUrl(self,url):
        '''
        删除重复url
        '''
        unrepect_url = []
        for l in self.sameTargetUrl(url):
            if l not in unrepect_url:
                unrepect_url.append(l)
        return unrepect_url

    def crawler(self,crawl_deepth=1):
        '''
        正式的爬取，并依据深度进行爬取层级控制
        '''
        while self.current_deepth <= crawl_deepth:
            while not self.linkQuence.unvisitedUrlEmpty():
                visitedUrl = self.linkQuence.popUnvisitedUrl()
                #print(visitedUrl)
                if visitedUrl is None or visitedUrl == '':
                    continue
                links = self.unrepectUrl(visitedUrl)
                self.linkQuence.addVisitedUrl(visitedUrl)
                for link in links:
                    self.linkQuence.addUnvisitedUrl(link)
            self.current_deepth += 1
        return self.linkQuence.visited


if __name__ == '__main__':
    spider = Spider(url)
    spider.crawler(1) #传递1就好了